1c80ba31b3c88437bfe5ffab2e6830e61a524292
3 * Import data from a UseModWiki into a MediaWiki wiki
4 * 2003-02-09 Brion VIBBER <brion@pobox.com>
5 * Based loosely on Magnus's code from 2001-2002
7 * Updated limited version to get something working temporarily
9 * Be sure to run the link & index rebuilding scripts!
11 * Some more munging for charsets etc
14 * Partial fix for pages starting with lowercase letters (??)
15 * and CamelCase and /Subpage link conversion
18 * Rewrite output to create Special:Export format for import
19 * instead of raw SQL. Should be 'future-proof' against future
23 * This program is free software; you can redistribute it and/or modify
24 * it under the terms of the GNU General Public License as published by
25 * the Free Software Foundation; either version 2 of the License, or
26 * (at your option) any later version.
28 * This program is distributed in the hope that it will be useful,
29 * but WITHOUT ANY WARRANTY; without even the implied warranty of
30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 * GNU General Public License for more details.
33 * You should have received a copy of the GNU General Public License along
34 * with this program; if not, write to the Free Software Foundation, Inc.,
35 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
36 * http://www.gnu.org/copyleft/gpl.html
40 * @ingroup Maintenance
43 require_once( "Maintenance.php" );
45 class ImportUseModWiki
extends Maintenance
{
47 private $encoding, $rootDirectory = '';
53 private $FS1, $FS2, $FS3 = '';
58 private $usercache, $nowiki = array();
60 public function __construct() {
61 parent
::__construct();
62 $this->mDescription
= "Import pages from UseMod wikis";
63 $this->addOption( 'encoding', 'Encoding of the imported text, default CP1252', false, true );
65 * If UseModWiki's New File System is used:
66 * $NewFS = 1; # 1 = new multibyte $FS, 0 = old $FS
67 * Use "\xb3"; for the Old File System
68 * Changed with UTF-8 UseModWiki
69 * http://www.usemod.com/cgi-bin/wiki.pl?SupportForUtf8
70 * http://www.usemod.com/cgi-bin/wiki.pl?WikiBugs/NewFieldSeparatorWronglyTreated
71 * http://www.meatballwiki.org/wiki/WikiEngine#Q_amp_A
73 $this->addOption( 'separator', 'Field separator to use, default \x1E\xFF\xFE\x1E', false, true );
74 $this->addArg( 'path', 'Path to your UseMod wiki' );
77 public function execute() {
78 $this->rootDirectory
= $this->getArg();
79 $this->encoding
= $this->getOption( 'encoding', 'CP1252' );
80 $sep = $this->getOption( 'separator', "\x1E\xFF\xFE\x1E" );
81 $this->FS1
= "{$sep}1";
82 $this->FS2
= "{$sep}2";
83 $this->FS3
= "{$sep}3";
86 <?xml version="1.0" encoding="UTF-8" ?>
87 <mediawiki xmlns="http://www.mediawiki.org/xml/export-0.1/"
88 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
89 xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.1/
90 http://www.mediawiki.org/xml/export-0.1.xsd"
93 <!-- generated by importUseModWiki.php -->
97 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
98 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
99 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'other' );
100 foreach ( $letters as $letter ) {
101 $dir = "{$this->rootDirectory}/page/$letter";
102 if ( is_dir( $dir ) )
103 $this->importPageDirectory( $dir );
111 private function importPageDirectory( $dir, $prefix = "" ) {
112 echo "\n<!-- Checking page directory " . $this->xmlCommentSafe( $dir ) . " -->\n";
113 $mydir = opendir( $dir );
114 while ( $entry = readdir( $mydir ) ) {
116 if ( preg_match( '/^(.+)\.db$/', $entry, $m ) ) {
117 echo $this->importPage( $prefix . $m[1] );
119 if ( is_dir( "$dir/$entry" ) ) {
120 if ( $entry != '.' && $entry != '..' ) {
121 $this->importPageDirectory( "$dir/$entry", "$entry/" );
124 echo "<!-- File '" . $this->xmlCommentSafe( $entry ) . "' doesn't seem to contain an article. Skipping. -->\n";
130 private function useModFilename( $title ) {
131 $c = substr( $title, 0, 1 );
132 if ( preg_match( '/[A-Z]/i', $c ) ) {
133 return strtoupper( $c ) . "/$title";
135 return "other/$title";
138 private function fetchPage( $title ) {
139 $fname = $this->rootDirectory
. "/page/" . $this->useModFilename( $title ) . ".db";
140 if ( !file_exists( $fname ) ) {
141 echo "Couldn't open file '$fname' for page '$title'.\n";
145 $page = $this->splitHash( $this->FS1
, file_get_contents( $fname ) );
146 $section = $this->splitHash( $this->FS2
, $page["text_default"] );
147 $text = $this->splitHash( $this->FS3
, $section["data"] );
149 return $this->array2object( array( "text" => $text["text"] , "summary" => $text["summary"] ,
150 "minor" => $text["minor"] , "ts" => $section["ts"] ,
151 "username" => $section["username"] , "host" => $section["host"] ) );
154 private function fetchKeptPages( $title ) {
155 $fname = $this->rootDirectory
. "/keep/" . $this->useModFilename( $title ) . ".kp";
156 if ( !file_exists( $fname ) ) return array();
158 $keptlist = explode( $this->FS1
, file_get_contents( $fname ) );
159 array_shift( $keptlist ); # Drop the junk at beginning of file
161 $revisions = array();
162 foreach ( $keptlist as $rev ) {
163 $section = $this->splitHash( $this->FS2
, $rev );
164 $text = $this->splitHash( $this->FS3
, $section["data"] );
165 if ( $text["text"] && $text["minor"] != "" && ( $section["ts"] * 1 > 0 ) ) {
166 array_push( $revisions, $this->array2object( array ( "text" => $text["text"] , "summary" => $text["summary"] ,
167 "minor" => $text["minor"] , "ts" => $section["ts"] ,
168 "username" => $section["username"] , "host" => $section["host"] ) ) );
170 echo "<!-- skipped a bad old revision -->\n";
176 private function splitHash( $sep , $str ) {
177 $temp = explode ( $sep , $str ) ;
179 for ( $i = 0; $i +
1 < count ( $temp ) ; $i++
) {
180 $ret[$temp[$i]] = $temp[++
$i] ;
185 private function checkUserCache( $name, $host ) {
187 if ( in_array( $name, $this->usercache
) ) {
188 $userid = $this->usercache
[$name];
190 # If we haven't imported user accounts
193 $username = str_replace( '_', ' ', $name );
198 return array( $userid, $username );
201 private function importPage( $title ) {
202 echo "\n<!-- Importing page " . $this->xmlCommentSafe( $title ) . " -->\n";
203 $page = $this->fetchPage( $title );
205 $newtitle = $this->xmlsafe( str_replace( '_', ' ', $this->recodeText( $title ) ) );
207 $munged = $this->mungeFormat( $page->text
);
208 if ( $munged != $page->text
) {
210 * Save a *new* revision with the conversion, and put the
211 * previous last version into the history.
213 $next = $this->array2object( array(
216 'username' => 'Conversion script',
217 'host' => '127.0.0.1',
219 'summary' => 'link fix',
221 $revisions = array( $page, $next );
226 $revisions = array( $page );
230 <title>$newtitle</title>
235 $revisions = array_merge( $revisions, $this->fetchKeptPages( $title ) );
236 if ( count( $revisions ) == 0 ) {
237 return NULL; // Was "$sql", which does not appear to be defined.
240 foreach ( $revisions as $rev ) {
241 $text = $this->xmlsafe( $this->recodeText( $rev->text
) );
242 $minor = ( $rev->minor ?
'<minor/>' : '' );
243 list( /* $userid */ , $username ) = $this->checkUserCache( $rev->username
, $rev->host
);
244 $username = $this->xmlsafe( $this->recodeText( $username ) );
245 $timestamp = $this->xmlsafe( $this->timestamp2ISO8601( $rev->ts
) );
246 $comment = $this->xmlsafe( $this->recodeText( $rev->summary
) );
250 <timestamp>$timestamp</timestamp>
251 <contributor><username>$username</username></contributor>
253 <comment>$comment</comment>
259 $xml .= "</page>\n\n";
263 private function recodeText( $string ) {
264 # For currently latin-1 wikis
265 $string = str_replace( "\r\n", "\n", $string );
266 $string = @iconv
( $this->encoding
, "UTF-8", $string );
267 $string = $this->mungeToUtf8( $string ); # Any old Ӓ stuff
272 * @fixme don't use /e
274 private function mungeToUtf8( $string ) {
275 $string = preg_replace ( '/&#([0-9]+);/e', 'wfUtf8Sequence($1)', $string );
276 $string = preg_replace ( '/&#x([0-9a-f]+);/ie', 'wfUtf8Sequence(0x$1)', $string );
277 # Should also do named entities here
281 private function timestamp2ISO8601( $ts ) {
282 # 2003-08-05T18:30:02Z
283 return gmdate( 'Y-m-d', $ts ) . 'T' . gmdate( 'H:i:s', $ts ) . 'Z';
287 * The page may contain old data which has not been properly normalized.
288 * Invalid UTF-8 sequences or forbidden control characters will make our
289 * XML output invalid, so be sure to strip them out.
290 * @param String $string Text to clean up
293 private function xmlsafe( $string ) {
294 $string = UtfNormal
::cleanUp( $string );
295 $string = htmlspecialchars( $string );
299 private function xmlCommentSafe( $text ) {
300 return str_replace( '--', '\\-\\-', $this->xmlsafe( $this->recodeText( $text ) ) );
303 private function array2object( $arr ) {
305 foreach ( $arr as $x => $y ) {
312 * Make CamelCase and /Talk links work
314 private function mungeFormat( $text ) {
315 $this->nowiki
= array();
316 $staged = preg_replace_callback(
317 '/(<nowiki>.*?<\\/nowiki>|(?:http|https|ftp):\\S+|\[\[[^]\\n]+]])/s',
318 array( $this, 'nowikiPlaceholder' ), $text );
320 # This is probably not 100% correct, I'm just
321 # glancing at the UseModWiki code.
323 $lower = "[a-z_0-9]";
324 $any = "[A-Za-z_0-9]";
325 $camel = "(?:$upper+$lower+$upper+$any*)";
326 $subpage = "(?:\\/$any+)";
327 $substart = "(?:\\/$upper$any*)";
329 $munged = preg_replace( "/(?!\\[\\[)($camel$subpage*|$substart$subpage*)\\b(?!\\]\\]|>)/",
332 $final = preg_replace( '/' . preg_quote( $this->placeholder() ) . '/s',
333 array( $this, 'nowikiShift' ), $munged );
337 private function placeholder( $x = null ) {
338 return '\xffplaceholder\xff';
341 public function nowikiPlaceholder( $matches ) {
342 $this->nowiki
[] = $matches[1];
343 return $this->placeholder();
346 public function nowikiShift() {
347 return array_shift( $this->nowiki
);
351 function wfUtf8Sequence( $codepoint ) {
352 if ( $codepoint < 0x80 ) {
353 return chr( $codepoint );
355 if ( $codepoint < 0x800 ) {
356 return chr( $codepoint >> 6 & 0x3f |
0xc0 ) .
357 chr( $codepoint & 0x3f |
0x80 );
359 if ( $codepoint < 0x10000 ) {
360 return chr( $codepoint >> 12 & 0x0f |
0xe0 ) .
361 chr( $codepoint >> 6 & 0x3f |
0x80 ) .
362 chr( $codepoint & 0x3f |
0x80 );
364 if ( $codepoint < 0x100000 ) {
365 return chr( $codepoint >> 18 & 0x07 |
0xf0 ) . # Double-check this
366 chr( $codepoint >> 12 & 0x3f |
0x80 ) .
367 chr( $codepoint >> 6 & 0x3f |
0x80 ) .
368 chr( $codepoint & 0x3f |
0x80 );
370 # Doesn't yet handle outside the BMP
371 return "&#$codepoint;";
374 $maintClass = 'ImportUseModWiki';
375 require_once( RUN_MAINTENANCE_IF_MAIN
);